{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed = 16\n",
    "import pandas as pd\n",
    "from gensim.models.word2vec import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read csv ended...\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv(\"ai_challenger_sentiment_analysis_trainingset_20180816/sentiment_analysis_trainingset.csv\")\n",
    "print(\"read csv ended...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>location_traffic_convenience</th>\n",
       "      <th>location_distance_from_business_district</th>\n",
       "      <th>location_easy_to_find</th>\n",
       "      <th>service_wait_time</th>\n",
       "      <th>service_waiters_attitude</th>\n",
       "      <th>service_parking_convenience</th>\n",
       "      <th>service_serving_speed</th>\n",
       "      <th>price_level</th>\n",
       "      <th>price_cost_effective</th>\n",
       "      <th>...</th>\n",
       "      <th>environment_decoration</th>\n",
       "      <th>environment_noise</th>\n",
       "      <th>environment_space</th>\n",
       "      <th>environment_cleaness</th>\n",
       "      <th>dish_portion</th>\n",
       "      <th>dish_taste</th>\n",
       "      <th>dish_look</th>\n",
       "      <th>dish_recommendation</th>\n",
       "      <th>others_overall_experience</th>\n",
       "      <th>others_willing_to_consume_again</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>105000.00000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>105000.00000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "      <td>105000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>52499.50000</td>\n",
       "      <td>-1.360267</td>\n",
       "      <td>-1.407095</td>\n",
       "      <td>-1.402276</td>\n",
       "      <td>-1.749895</td>\n",
       "      <td>-0.496495</td>\n",
       "      <td>-1.846952</td>\n",
       "      <td>-1.661457</td>\n",
       "      <td>-0.975800</td>\n",
       "      <td>-1.379238</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.67160</td>\n",
       "      <td>-1.203162</td>\n",
       "      <td>-1.065410</td>\n",
       "      <td>-1.033552</td>\n",
       "      <td>-0.907552</td>\n",
       "      <td>0.389181</td>\n",
       "      <td>-1.275771</td>\n",
       "      <td>-1.484181</td>\n",
       "      <td>0.537771</td>\n",
       "      <td>-0.981248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>30311.03347</td>\n",
       "      <td>1.210342</td>\n",
       "      <td>1.185697</td>\n",
       "      <td>1.145902</td>\n",
       "      <td>0.739263</td>\n",
       "      <td>1.358622</td>\n",
       "      <td>0.618700</td>\n",
       "      <td>0.866716</td>\n",
       "      <td>1.151532</td>\n",
       "      <td>1.166656</td>\n",
       "      <td>...</td>\n",
       "      <td>1.41387</td>\n",
       "      <td>1.266776</td>\n",
       "      <td>1.282662</td>\n",
       "      <td>1.337833</td>\n",
       "      <td>1.308446</td>\n",
       "      <td>0.781373</td>\n",
       "      <td>1.224178</td>\n",
       "      <td>1.095536</td>\n",
       "      <td>0.740198</td>\n",
       "      <td>1.372485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.00000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>26249.75000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.00000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>52499.50000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.00000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>78749.25000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>104999.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id  location_traffic_convenience  \\\n",
       "count  105000.00000                 105000.000000   \n",
       "mean    52499.50000                     -1.360267   \n",
       "std     30311.03347                      1.210342   \n",
       "min         0.00000                     -2.000000   \n",
       "25%     26249.75000                     -2.000000   \n",
       "50%     52499.50000                     -2.000000   \n",
       "75%     78749.25000                     -2.000000   \n",
       "max    104999.00000                      1.000000   \n",
       "\n",
       "       location_distance_from_business_district  location_easy_to_find  \\\n",
       "count                             105000.000000          105000.000000   \n",
       "mean                                  -1.407095              -1.402276   \n",
       "std                                    1.185697               1.145902   \n",
       "min                                   -2.000000              -2.000000   \n",
       "25%                                   -2.000000              -2.000000   \n",
       "50%                                   -2.000000              -2.000000   \n",
       "75%                                   -2.000000              -2.000000   \n",
       "max                                    1.000000               1.000000   \n",
       "\n",
       "       service_wait_time  service_waiters_attitude  \\\n",
       "count      105000.000000             105000.000000   \n",
       "mean           -1.749895                 -0.496495   \n",
       "std             0.739263                  1.358622   \n",
       "min            -2.000000                 -2.000000   \n",
       "25%            -2.000000                 -2.000000   \n",
       "50%            -2.000000                  0.000000   \n",
       "75%            -2.000000                  1.000000   \n",
       "max             1.000000                  1.000000   \n",
       "\n",
       "       service_parking_convenience  service_serving_speed    price_level  \\\n",
       "count                105000.000000          105000.000000  105000.000000   \n",
       "mean                     -1.846952              -1.661457      -0.975800   \n",
       "std                       0.618700               0.866716       1.151532   \n",
       "min                      -2.000000              -2.000000      -2.000000   \n",
       "25%                      -2.000000              -2.000000      -2.000000   \n",
       "50%                      -2.000000              -2.000000      -2.000000   \n",
       "75%                      -2.000000              -2.000000       0.000000   \n",
       "max                       1.000000               1.000000       1.000000   \n",
       "\n",
       "       price_cost_effective               ...                 \\\n",
       "count         105000.000000               ...                  \n",
       "mean              -1.379238               ...                  \n",
       "std                1.166656               ...                  \n",
       "min               -2.000000               ...                  \n",
       "25%               -2.000000               ...                  \n",
       "50%               -2.000000               ...                  \n",
       "75%               -2.000000               ...                  \n",
       "max                1.000000               ...                  \n",
       "\n",
       "       environment_decoration  environment_noise  environment_space  \\\n",
       "count            105000.00000      105000.000000      105000.000000   \n",
       "mean                 -0.67160          -1.203162          -1.065410   \n",
       "std                   1.41387           1.266776           1.282662   \n",
       "min                  -2.00000          -2.000000          -2.000000   \n",
       "25%                  -2.00000          -2.000000          -2.000000   \n",
       "50%                  -2.00000          -2.000000          -2.000000   \n",
       "75%                   1.00000           0.000000           0.000000   \n",
       "max                   1.00000           1.000000           1.000000   \n",
       "\n",
       "       environment_cleaness   dish_portion     dish_taste      dish_look  \\\n",
       "count         105000.000000  105000.000000  105000.000000  105000.000000   \n",
       "mean              -1.033552      -0.907552       0.389181      -1.275771   \n",
       "std                1.337833       1.308446       0.781373       1.224178   \n",
       "min               -2.000000      -2.000000      -2.000000      -2.000000   \n",
       "25%               -2.000000      -2.000000       0.000000      -2.000000   \n",
       "50%               -2.000000      -2.000000       1.000000      -2.000000   \n",
       "75%                1.000000       1.000000       1.000000      -1.000000   \n",
       "max                1.000000       1.000000       1.000000       1.000000   \n",
       "\n",
       "       dish_recommendation  others_overall_experience  \\\n",
       "count        105000.000000              105000.000000   \n",
       "mean             -1.484181                   0.537771   \n",
       "std               1.095536                   0.740198   \n",
       "min              -2.000000                  -2.000000   \n",
       "25%              -2.000000                   0.000000   \n",
       "50%              -2.000000                   1.000000   \n",
       "75%              -2.000000                   1.000000   \n",
       "max               1.000000                   1.000000   \n",
       "\n",
       "       others_willing_to_consume_again  \n",
       "count                    105000.000000  \n",
       "mean                         -0.981248  \n",
       "std                           1.372485  \n",
       "min                          -2.000000  \n",
       "25%                          -2.000000  \n",
       "50%                          -2.000000  \n",
       "75%                           1.000000  \n",
       "max                           1.000000  \n",
       "\n",
       "[8 rows x 21 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(data.describe()\n",
    "print(data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get stop word list. ended...\n"
     ]
    }
   ],
   "source": [
    "stopwords = []\n",
    "with open(\"stopwords.txt\") as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip()\n",
    "        stopwords.append(line)\n",
    "print(\"get stop word list. ended...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def segWord(doc):\n",
    "    seg_list = jieba.cut(doc, cut_all=False)\n",
    "    return list(seg_list)\n",
    "\n",
    "#move stop words\n",
    "def filter_map(arr):\n",
    "    res = \"\"\n",
    "    for c in arr:\n",
    "        if c not in stopwords and c != ' ' and c != '\\xa0'and c != '\\n' and c != '\\ufeff' and c != '\\r':\n",
    "            res += c\n",
    "    return res\n",
    "  \n",
    "#move stop words and generate char sent\n",
    "def filter_char_map(arr):\n",
    "    res = []\n",
    "    for c in arr:\n",
    "        if c not in stopwords and c != ' ' and c != '\\xa0'and c != '\\n' and c != '\\ufeff' and c != '\\r':\n",
    "            res.append(c)\n",
    "    return \" \".join(res)\n",
    "#get char of sentence\n",
    "def get_char(arr):\n",
    "    res = []\n",
    "    for c in arr:\n",
    "        res.append(c)\n",
    "    return list(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "started...\n",
      "fileter and tokenize to char list. ended...\n"
     ]
    }
   ],
   "source": [
    "print(\"started...\")\n",
    "data.content = data.content.map(lambda x: filter_map(x))\n",
    "data.content = data.content.map(lambda x: get_char(x))\n",
    "print(\"fileter and tokenize to char list. ended...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    [吼, 吼, 吼, 萌, 死, 人, 棒, 棒, 糖, 中, 大, 众, 点, 评, 霸, ...\n",
      "1    [三, 次, 参, 加, 大, 众, 点, 评, 网, 霸, 王, 餐, 活, 动, 家, ...\n",
      "2    [人, 行, 点, 小, 吃, 榴, 莲, 酥, 榴, 莲, 味, 道, 不, 足, 松, ...\n",
      "3    [前, 评, 价, 莫, 名, 妙, 删, 果, 断, 继, 续, 差, 评, 换, 菜, ...\n",
      "4    [出, 意, 料, 惊, 艳, 椰, 子, 鸡, 清, 热, 降, 火, 美, 容, 养, ...\n",
      "Name: content, dtype: object\n",
      "end...\n"
     ]
    }
   ],
   "source": [
    "print(data[\"content\"].head())\n",
    "print(\"end...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "end...\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(\"preprocess/train_char.csv\", index=None)\n",
    "print(\"end...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train word2vec ended..\n"
     ]
    }
   ],
   "source": [
    "line_sent = []\n",
    "for s in data[\"content\"]:\n",
    "    line_sent.append(s)\n",
    "word2vec_model = Word2Vec(line_sent, size=100, window=10, min_count=1, workers=4, iter=15)\n",
    "word2vec_model.wv.save_word2vec_format(\"word2vec/chars.vector\", binary=True)\n",
    "print(\"train word2vec ended..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "process validation set ended..\n"
     ]
    }
   ],
   "source": [
    "validation = pd.read_csv(\"ai_challenger_sentiment_analysis_validationset_20180816/sentiment_analysis_validationset.csv\")\n",
    "validation.content = validation.content.map(lambda x: filter_map(x))\n",
    "validation.content = validation.content.map(lambda x: get_char(x))\n",
    "validation[\"content\"].head()\n",
    "print(\"process validation set ended..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "write validation set.end...\n"
     ]
    }
   ],
   "source": [
    "validation.to_csv(\"preprocess/validation_char.csv\", index=None)\n",
    "print(\"write validation set.end...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read test set.end...\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv(\"ai_challenger_sentiment_analysis_testa_20180816/sentiment_analysis_testa.csv\")\n",
    "print(\"read test set.end...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "write test set.end...\n"
     ]
    }
   ],
   "source": [
    "test.content = test.content.map(lambda x: filter_map(x))\n",
    "test.content = test.content.map(lambda x: get_char(x))\n",
    "test.to_csv(\"preprocess/test_char.csv\", index=None)\n",
    "print(\"write test set.end...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
